#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <time.h>

/************************************************************************/
/*                            ��������                                  */
/************************************************************************/
float absolute(float a)
{
	if (a < 0)
	{
		return -a;
	}
	else
	{
		return a;
	}
}


float min(float a, float b)
{
	if (a > b)
	{
		return b;
	}
	else
	{
		return a;
	}
}


int equal(float a, float b)
{
	if (absolute(a - b)<0.00000001)
	{
		return 1;
	}
	else
	{
		return 0;
	}
}


char* F2S(float f, char* str)
{
	char str1[40];
	int j = 0, k, i;
	float d;

	d = absolute(f);
	i = (int)d;//����������������
	while (i > 0)
	{
		str1[j++] = i % 10 + '0';
		i = i / 10;
	}
	for (k = 0; k < j; k++)
	{
		str[k] = str1[j - 1 - k];//����ȡ�����������������ŵ���һ������
	}

	if ((int)d == 0)
	{
		str[j++] = '0';
	}

	str[j++] = '.';

	d = d - (int)d;//С��������ȡ
				   //printf("С������%f\n", d);
	for (i = 0; i < 4; i++)
	{
		d = d * 10;
		str[j++] = (int)d + '0';
		//printf("%c", str[j - 1]);
		d = d - (int)d;

	}
	/*while (str[--j] == '0');*/
	str[j] = '\0';

	/*printf("%c\n", str[0]);*/

	//��������
	if (f < 0)
	{
		j = 0;
		while (str[j] != '\0')
		{
			++j;
		}
		str[j + 1] = '\0';
		while (j > 0)
		{
			str[j] = str[j - 1];
			--j;
		}
		str[0] = '-';
	}
	return str;
}
/************************************************************************/
/*                            ��������                                  */
/************************************************************************/

typedef struct {
	int row, col;
	float **element;
}Mat;

/************************************************************************/
/*                              Mat����                                  */
/************************************************************************/
Mat* MatCreate(Mat* mat, int row, int col)
{
	int i;

#ifdef MAT_LEGAL_CHECKING
	if (row <= 0 || col <= 0) {
		printf("err check, unmatch matrix for MatCreate\n");
		printf("\t\trow:\n\t\t\t");
		printf("%d\n", row);
		printf("\t\tcol:\n\t\t\t");
		printf("%d\n", col);
		return NULL;
	}
#endif


	mat->element = (float**)malloc(row * sizeof(float*));
	if (mat->element == NULL) {
		printf("mat create fail! size:[%d,%d]\n", row, col);
		return NULL;
	}
	for (i = 0; i < row; i++) {
		mat->element[i] = (float*)malloc(col * sizeof(float));
		if (mat->element[i] == NULL) {
			int j;
			printf("mat create fail! size:[%d,%d],spot:[%d]\n", row, col, i);
			for (j = 0; j < i; j++)
				free(mat->element[j]);
			free(mat->element);
			return NULL;
		}
	}

	mat->row = row;
	mat->col = col;

	//printf("Mat create succeed, addr:%p\n", &mat->element);
	return mat;
}


void MatDelete(Mat* mat)
{
	int i;

	for (i = 0; i < mat->row; i++)
		free(mat->element[i]);
	free(mat->element);
}


Mat* MatSetVal(Mat* mat, float* val)
{
	int row, col;

	for (row = 0; row < mat->row; row++) {
		for (col = 0; col < mat->col; col++) {
			(mat->element[row])[col] = val[col + row * mat->col];
		}
	}

	return mat;
}


void MatShape(const Mat* mat)
{
#ifdef MAT_LEGAL_CHECKING
	if (mat == NULL) {
		printf("err check for MatShape\n");
		return;
	}
#endif

	printf("Mat %dx%d:\n", mat->row, mat->col);
}


void MatDump(const Mat* mat)
{
	int row, col;
	char str[40];
	char * data;

#ifdef MAT_LEGAL_CHECKING
	if (mat == NULL) {
		printf("err check for MatDump\n");
		return;
	}
#endif

	printf("Mat %dx%d:\n", mat->row, mat->col);
	for (row = 0; row < mat->row; row++) {
		for (col = 0; col < mat->col; col++) {
			data = F2S((mat->element[row])[col], str);
			if (data[0] == '-') {
				printf(" %s\t", F2S((mat->element[row])[col], str));
			}
			else {
				printf("  %s\t", F2S((mat->element[row])[col], str));
			}

		}
		printf("\n");
	}
	printf("\n");
	//for (row = 0; row < mat->row; row++) {
	//	for (col = 0; col < mat->col; col++) {
	//		data = (mat->element[row])[col];
	//		if (data<0){
	//			printf("%.4f\t", data);
	//		}
	//		else{
	//			printf(" %.4f\t", data);
	//		}

	//	}
	//	printf("\n");
	//}
	//printf("\n");
}

/*ȫ0��*/
Mat* MatZeros(Mat* mat)
{
	int row, col;

	for (row = 0; row < mat->row; row++) {
		for (col = 0; col < mat->col; col++) {
			(mat->element[row])[col] = 0.0f;
		}
	}

	return mat;
}



/*ȫ1��*/
Mat* MatOnes(Mat* mat)
{
	int row, col;

	for (row = 0; row < mat->row; row++) {
		for (col = 0; col < mat->col; col++) {
			(mat->element[row])[col] = 1.0f;
		}
	}

	return mat;
}


/*�Խ�1����*/
Mat* MatEye(Mat* mat)
{
	int i;

	MatZeros(mat);
	for (i = 0; i < min(float(mat->row), float(mat->col)); i++) {
		(mat->element[i])[i] = 1.0f;
	}

	return mat;
}


/* dst = src1 + src2 */
Mat* MatAdd(Mat* src1, Mat* src2, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (!(src1->row == src2->row && src2->row == dst->row && src1->col == src2->col && src2->col == dst->col)) {
		printf("\t\terr check, unmatch matrix for MatAdd\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src1);
		printf("\t\t\t");
		MatShape(src2);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src1->row; row++) {
		for (col = 0; col < src1->col; col++) {
			(dst->element[row])[col] = (src1->element[row])[col] + (src2->element[row])[col];
		}
	}

	return dst;
}


/* dst = src1 - src2 */
Mat* MatSub(Mat* src1, Mat* src2, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (!(src1->row == src2->row && src2->row == dst->row && src1->col == src2->col && src2->col == dst->col)) {
		printf("\t\terr check, unmatch matrix for MatSub\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src1);
		printf("\t\t\t");
		MatShape(src2);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src1->row; row++) {
		for (col = 0; col < src1->col; col++) {
			(dst->element[row])[col] = (src1->element[row])[col] - (src2->element[row])[col];
		}
	}

	return dst;
}


/* dst = src1 x src2  ϡ���Ĵ��������Ż�*/
Mat* MatMul(Mat* src1, Mat* src2, Mat* dst)
{
	int row, col;
	int i;

#ifdef MAT_LEGAL_CHECKING
	if (src1->col != src2->row || src1->row != dst->row || src2->col != dst->col) {
		printf("\t\terr check, unmatch matrix for MatMul\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src1);
		printf("\t\t\t");
		MatShape(src2);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif
	MatZeros(dst);
	float temp1 = 0.f;
	float temp2 = 0.f;
	for (row = 0; row < src1->row; row++) {
		for (col = 0; col < src1->col; col++) {
			temp1 = (src1->element[row])[col];
			if (equal(temp1, 0) == 0) {
				for (i = 0; i < src2->col; i++) {
					temp2 = (src2->element[col])[i];
					if (equal(temp2, 0) == 0) {
						(dst->element[row])[i] += temp1 * temp2;
					}
				}
			}
		}
	}

	return dst;
}


/* dst = src1 x src2 ԭʼ */
Mat* MatMul2(Mat* src1, Mat* src2, Mat* dst)
{
	int row, col;
	int i;
	float temp;

#ifdef MAT_LEGAL_CHECKING
	if (src1->col != src2->row || src1->row != dst->row || src2->col != dst->col) {
		printf("\t\terr check, unmatch matrix for MatMul\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src1);
		printf("\t\t\t");
		MatShape(src2);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < dst->row; row++) {
		for (col = 0; col < dst->col; col++) {
			for (i = 0; i < src1->col; i++) {
				(dst->element[row])[col] += (src1->element[row])[i] * (src2->element[i])[col];
			}
		}
	}

	return dst;
}
////2.1 �Ӿ����˷� C=A'*B
//void SMblock_MultCAOB(Mat *rawCAOB, Mat rawA, Mat rawB, int si, int sj, int sk, int subm, int subn, int subp, int T, int S) {
//	int i, j, k;
//	for (j = 0; j < subn; j++) { //�к�
//		for (i = 0; i < subm; i++) { //�к�
//			for (k = 0; k < subp; k++) { //����
//										 //printf("�ӿ��ˣ�C[%d][%d]+=A[%d][%d]*B[%d][%d] \n",sj * T + j,sk * S + k,si*S + i,sj * T + j,si * T + i,sk*S + k);
//										 //C[j * p + k]+= A[i*m + j] * B[i * p + k];  //�ο�
//				(rawCAOB->element[sj * T + j])[sk * T + k] += (rawA.element[si*S + i])[sj * T + j] * (rawB.element[si*S + i])[sk * T + k];
//
//			}
//		}
//	}
//}
//
//
///* dst = src1 x src2 �����Ż�*/
//Mat* MatMul2(Mat* src1, Mat* src2, Mat* dst)
//{
//
//#ifdef MAT_LEGAL_CHECKING
//	if (src1->col != src2->row || src1->row != dst->row || src2->col != dst->col) {
//		printf("\t\terr check, unmatch matrix for MatMul\n");
//		printf("\t\tsrcMatShape:\n\t\t\t");
//		MatShape(src1);
//		printf("\t\t\t");
//		MatShape(src2);
//		printf("\t\tdstMatShape:\n\t\t\t");
//		MatShape(dst);
//		return NULL;
//	}
//#endif
//
//	int rawm = src1->row;
//	int rawn = src2->col;
//	int rawp = src2->col;
//
//	// �Ժ�����
//	int S = 1;  //����������
//	int T = 2;  //����������
//
//	//�ֿ����Ӿ����ĸ���h*l,A������ΪS*T���Ӿ�����B������ΪT*S���Ӿ���
//	static int col_M = 1;
//	static int row_N = 1;
//
//	//����A��B�ֿ��󣬲���ȫ�ֿ�ʱ������һ�к�����һ�е��Ӿ����Ĵ�С
//	static int col_last = 1;
//	static int row_last = 1;
//
//	//====================================================================================
//	//������rawA[rawm][rawn]��ΪC_M*R_N����СΪS*T���ӿ飬ceil(x)�������ز�С��x����С����
//	if (rawm % S == 0) {
//		col_M = rawm / S;
//	}
//	else {
//		col_M = rawm / S + 1;
//	}
//	//AC_M = ceil((double) rawm / (double) S); //����A�ֿ���������
//	row_N = ceil((double)rawn / (double)T); //����A�ֿ�����������������B�ֿ���������
//	col_last = rawm - (col_M - 1) * S;//����һ��
//	row_last = rawn - (row_N - 1) * T;//����һ��
//	printf("%d\n", row_N);
//	printf("%d\n", col_M);
//	printf("%d\n", row_last);
//	printf("%d\n", col_last);
//	//====================================================================================
//	//MatDump(dst);
//	int i, j, k;
//	int count = 0;//ѭ������
//				  //ѭ�� ��˳���ɸ�����Ҫ����������Ӱ�������Ľ���
//	for (j = 0; j < row_N; j++) {
//		for (i = 0; i < col_M; i++) {
//			for (k = 0; k < row_N; k++) {
//								printf("\t ��%d��ѭ����  ",++count);
//				//				printf(" �ֿ��˷���C[%d][%d]+=A[%d][%d]*B[%d][%d] \n",j,k,i,j,i,k);
//				int mblk = S, nblk = T, pblk = T;//Ĭ�ϵ�ǰ���������������Ӿ������Ĵ�С������ÿ��ѭ�����¸���ֵ
//												 //���㵱ǰ�ӿ��Ĵ�СΪmblk*nblk
//				if ((i == col_M - 1)) {
//					mblk = col_last;
//				}
//				if (j == row_N - 1) {
//					nblk = row_last;
//				}
//				if (k == row_N - 1) {
//					pblk = row_last;
//				}
//
//				//�ֿ������˷�C=A'*B
//				//SMblock_MultCAOB(i, j, k, mblk, nblk, mblk);
//				SMblock_MultCAOB(dst, *src1, *src2, i, j, k, mblk, nblk, pblk, T, S);
//			}
//		}
//	}
//
//	return dst;
//}


/* dst = src1 * src2 */   // Hadamard product
Mat* MatProduct(Mat* src1, Mat* src2, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (!(src1->row == src2->row && src2->row == dst->row && src1->col == src2->col && src2->col == dst->col)) {
		printf("\t\terr check, unmatch matrix for MatAdd\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src1);
		printf("\t\t\t");
		MatShape(src2);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src1->row; row++) {
		for (col = 0; col < src1->col; col++) {
			(dst->element[row])[col] = (src1->element[row])[col] * (src2->element[row])[col];
		}
	}

	return dst;
}


/* dst = src1 / src2 */
Mat* MatDiv(Mat* src1, Mat* src2, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (!(src1->row == src2->row && src2->row == dst->row && src1->col == src2->col && src2->col == dst->col)) {
		printf("\t\terr check, unmatch matrix for MatDiv\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src1);
		printf("\t\t\t");
		MatShape(src2);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src1->row; row++) {
		for (col = 0; col < src1->col; col++) {
			(dst->element[row])[col] = (src1->element[row])[col] / (src2->element[row])[col];
		}
	}

	return dst;
}


/* dst = num * src ������������ */
Mat* MatNumMul(float num, Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatNumMul\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = num * (src->element[row])[col];
	}

	return dst;
}


/* dst = num + src ������������ */
Mat* MatNumAdd(float num, Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatNumAdd\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = num + (src->element[row])[col];
	}

	return dst;
}


/* dst = src^T */
Mat* MatTrans(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->col || src->col != dst->row) {
		printf("err check, unmatch matrix for MatTranspose\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < dst->row; row++) {
		for (col = 0; col < dst->col; col++) {
			(dst->element[row])[col] = (src->element[col])[row];
		}
	}

	return dst;
}






/*dst = sum(src)  src ������ÿһ������*/
Mat* MatRowSum(Mat* src, Mat* dst)
{
	int row, col;
	float temp;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || dst->col != 1) {
		printf("err check, unmatch matrix for MatSum\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		temp = 0;
		for (col = 0; col < src->col; col++) {
			temp += (src->element[row])[col];
			//printf("%f\t", temp);
		}
		(dst->element[row])[0] = temp;
		//printf("%f\n", temp);
	}

	return dst;
}


/*dst = MatMax(src)  src �ҳ�������ÿһ������ֵ*/
Mat* MatRowMax(Mat* src, Mat* dst)
{
	int row, col;
	float temp;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || dst->col != 1) {
		printf("err check, unmatch matrix for MatMax\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		temp = (src->element[row])[0];
		for (col = 1; col < src->col; col++) {
			if ((src->element[row])[col] > temp) {
				temp = (src->element[row])[col];
			}
		}
		(dst->element[row])[0] = temp;
	}

	return dst;
}

/* dst = src * src (Hadamard product)*/
Mat* MatSquare(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("err check, unmatch matrix for MatSquare\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			(dst->element[row])[col] = (src->element[row])[col] * (src->element[row])[col];
		}
	}

	return dst;
}


/* dst = Sqrt(src) */
Mat* MatSqrt(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("err check, unmatch matrix for MatSqrt\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			(dst->element[row])[col] = sqrt(float((src->element[row])[col]));
		}
	}

	return dst;
}


/*dst = MatExp(src)  ָ������*/
Mat* MatExp(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("err check, unmatch matrix for MatExp\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			(dst->element[row])[col] = (float)(exp(float((src->element[row])[col])));
		}
	}

	return dst;
}




/* dst = src - vector   ���������� �м�*/
Mat* MatVectorSub(Mat* src, Mat* vector, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col || vector->row != src->row || vector->col != 1) {
		printf("err check, unmatch matrix for MatVectorSub\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tvectorShape:\n\t\t\t");
		MatShape(vector);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif


	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			(dst->element[row])[col] = (src->element[row])[col] - (vector->element[row])[0];
		}
	}

	return dst;

}


/* dst = src / vector   ���������� �г�*/
Mat* MatVectorDiv(Mat* src, Mat* vector, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col || vector->row != src->row || vector->col != 1) {
		printf("err check, unmatch matrix for MatVectorSub\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tvectorShape:\n\t\t\t");
		MatShape(vector);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}

	for (row = 0; row < src->row; row++) {
		if (absolute((vector->element[row])[0]) < 0.000001) {
			printf("err check, Divisor vector has zero.\n");
			return NULL;
		}
	}
#endif


	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			(dst->element[row])[col] = (src->element[row])[col] / (vector->element[row])[0];
		}
	}

	return dst;

}


/* dst = src �ڴ濽�� */
void MatCopy(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatCopy\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = (src->element[row])[col];
	}
}

//--------------------------------Matplus--------------------------------//

/* dst = src^+   col+1*/
void MatPlusCol(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || (src->col) + 1 != dst->col) {
		printf("\t\terr check, unmathed matrix for MatPlus\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return;
	}
#endif
	for (row = 0, col = 0; row < dst->row; row++) {
		(dst->element[row])[col] = 1;
	}
	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col + 1] = (src->element[row])[col];
	}
}



/* dst = src^+    row+1*/
void MatPlusRow(Mat* src, Mat* dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row + 1 != dst->row || (src->col) != dst->col) {
		printf("\t\terr check, unmathed matrix for MatPlus\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return;
	}
#endif
	for (row = 0, col = 0; col < dst->col; col++) {
		(dst->element[row])[col] = 0.f;
	}
	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row + 1])[col] = (src->element[row])[col];
	}
}

// w:Constant +1
void MatPlusRowConstant(Mat* src,Mat* dst,float cst)
{
	int row,col;

	#ifdef MAT_LEGAL_CHECKING
	if (src->row + 1 != dst->row || (src->col) != dst->col) {
		printf("\t\terr check, unmathed matrix for MatPlus\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return;
	}
#endif
	for (row = 0, col = 0; col < dst->col; col++) {
		(dst->element[row])[col] = cst;
	}
	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row + 1])[col] = (src->element[row])[col];
	}
}

// w:Random Initialization
void MatReduceRow(Mat* src,Mat* dst)
{
	int row,col;

	#ifdef MAT_LEGAL_CHECKING
	if (src->row - 1 != dst->row || (src->col) != dst->col) {
		printf("\t\terr check, unmathed matrix for MatPlus\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return;
	}
#endif
	for (row = 1; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row - 1])[col] = (src->element[row])[col];
	}
}

/* dst = src1 * src2 */   // Hadamard product
Mat* MatplusProduct(Mat* src1, Mat* src2, Mat* dst, int hgt)
{
	int i, j, k;
	for (i = 0; i < hgt; ++i)
	{
		for (j = 0; j < dst[0].row; ++j)
		{
			for (k = 0; k < dst[0].col; ++k)
			{
				(dst[i].element[j])[k] = (src1[i].element[j])[k] * (src2[i].element[j])[k];
			}
		}
	}
	return dst;
}

void MatplusShape(Mat* mat, int mat_h)
{
#ifdef MAT_LEGAL_CHECKING
	if (mat == NULL) {
		printf("err check for MatplusShape\n");
		return;
	}
#endif

	printf("Mat %dx%dx%d:\n", mat_h, mat->row, mat->col);
}

Mat* MatplusZeros(Mat* des, int des_h)
{
	int i;
	for (i = 0; i < des_h; ++i)
	{
		MatZeros(&des[i]);
	}
}


//--------------------------------Matplus--------------------------------//


/************************************************************************/
/*                              Mat����                                  */
/************************************************************************/


typedef struct {
	int LayerType;				// Layer Type:0 for full-connection, 1 for convolution, 2 for deconvolution, 3 for max-pool, 4 for average-pool [int]
	int PaddingMode;			// Padding Mode:0 for VALID ,1 for SAME [int]
	int ActiFuncNum;			// Active Function for Conv: 1 for sigmoid, 2 for tanh, 3 for relu, 4 for l-relu, 5 for softmax [int]
	int KernelSize[2];			// Size of Kernel [k1,k2] [int[2]]
	int Stride[2];				// Operation Stride [weight,height] [int[2]] // could be 4
	Mat* OriginMat;				// Result of Padding of the Input [Mat*] with RC
	//Mat* ReConstruct;			// Reconstruct Mat for Operation (with bias for concolution) [Mat*]
	Mat KernelMat;				// Operation Kernel of this layer [Mat]
	Mat WeightMat;				// Kernel with bias for Concolution [Mat]
	Mat* SumMat;				// Sum Mat of this Layer [Mat*]
	Mat* ActiMat;				// Activate Sum Mat of Concolution [Mat*]
	Mat* DeriMat;				// Derication Activate Mat of Concolution [Mat*]
	Mat* DeltaMat;				// Error Delever Mat [Mat*]
	Mat NablaMat;				// Error on Weight Mat [Mat*]
	//int OutSize[3];				// Size of output [channel,height,weight] [int[4]]
	int OutH;					// Height of ActiMat
	int OutW;					// Weight of ActiMat
	int OutC;					// Channel of ActiMat
}Layer;

typedef struct {
	int sampleCapacity;         // number of current samples [int]
	int SampleRow;				// row(features) of sample [int]
	int SampleCol;				// column(features) of sample [int]
	int SampleCha;				// channels(features) of sample [int]
	int HiddenLayerNum;         // number of hidden layer [int]
	int WeightInitWayNum;       // weight initialization mode [int]
	//int *LayerType;				// layer type of each layers,  [int*]
	//int ParameterTotal;			// number of Parameters in this network [int]
	//int *LayerParameters;		// Parameters of each layer: fc with 1 para, cp with 8 paras in order of padding,k1,k2,outchannels,stride[4]
	/*int BatchNum;
	int BatchSize;
	int reminder;*/

	Layer *Layers;				// layers of FCNN
	Mat OnehotMat;              // onehot code Matrix

	//int *Optimization;			// Optimization Method chosen and its parameters[int*]
	int ClassificationNum;      // Number of categories classified [int]
	int LossFuncNum;            // loss function [int]
}CNN;


void InitLayer(Layer *ncp)
{
	ncp->LayerType = -1;
	ncp->PaddingMode = -1;
	ncp->ActiFuncNum = -1;
	ncp->KernelSize[0] = -1; ncp->KernelSize[1] = -1; ncp->KernelSize[2] = -1; ncp->KernelSize[3] = -1;
	ncp->Stride[0] = -1; ncp->Stride[1] = -1;// ncp->Stride[2] = -1; ncp->Stride[3] = -1;
	ncp->OriginMat = NULL;
	//ncp->ReConstruct = NULL;
	ncp->KernelMat.element = NULL;
	ncp->WeightMat.element = NULL;
	ncp->SumMat = NULL;
	ncp->ActiMat = NULL;
	ncp->DeriMat = NULL;
	//ncp->OutSize[0] = -1; ncp->OutSize[1] = -1; ncp->OutSize[2] = -1;// ncp->OutSize[3] = -1;
	ncp->OutH = -1; ncp->OutW = -1; ncp->OutC = -1;
}

void InitCNN(CNN *cnn)
{
	cnn->sampleCapacity = -1;
	cnn->SampleCol = -1;
	cnn->SampleRow = -1;
	cnn->SampleCha = -1;
	//cnn->LayerNum = -1;
	cnn->HiddenLayerNum = -1;
	cnn->WeightInitWayNum = -1;
	//cnn->ParameterTotal = -1;
	//cnn->LayerType = NULL;
	//cnn->LayerParameters = NULL;
	/*cnn->BatchNum = -1;
	cnn->BatchSize = -1;
	cnn->reminder = -1;*/
	cnn->Layers = NULL;
	cnn->OnehotMat.element = NULL;
}

/************************************************************************/
/*                            ��ʼ������                                 */
/************************************************************************/

/************************************************************************/
/*                             dump����                                  */
/************************************************************************/

void DumpIntArray(int* array, int n) {
	int i;
	for (i = 0; i < n; ++i) {
		printf("%d\t", array[i]);
	}
	printf("\n");
}

void DumpFloatArray(float* array, int n) {
	char str[40];
	int i;
	for (i = 0; i < n; ++i) {
		printf("%s", F2S(array[i], str));
		printf("\t");
	}
	printf("\n");
}

int DumpInputLayer(Layer in)
{
	printf("\tInput:\t\t[%d,%d,%d]\n", in.OutH, in.OutW, in.OutC);
	printf("ActiMat[%d,%d]\n", in.ActiMat[0].row, in.ActiMat[0].col);
	return 0;
}

int DumpFullConnLayer(Layer fc)
{
	printf("Layers:\t NeuronNum \t ActiNum\n");
	printf("   FcLayer\t %d \t\t %d \n", fc.OutC, fc.ActiFuncNum);
	printf("OriginMat[%d,%d]\n", fc.OriginMat[0].row, fc.OriginMat[0].col);
	printf("KernelMat[%d,%d]\tWeightMat[%d,%d]\tNablaMat[%d,%d]\n",
		fc.KernelMat.row, fc.KernelMat.col, fc.WeightMat.row, fc.WeightMat.col,
		fc.NablaMat.row, fc.NablaMat.col);
	printf("SumMat[%d,%d]\tActiMat[%d,%d]\tDeriMat[%d,%d]\tDeltaMat[%d,%d]\n",
		fc.SumMat[0].row, fc.SumMat[0].col, fc.ActiMat[0].row, fc.ActiMat[0].col,
		fc.DeriMat[0].row, fc.DeriMat[0].col, fc.DeltaMat[0].row, fc.DeltaMat[0].col);
	return 0;
}

int DumpConvLayer(Layer cv)
{
	printf("Layers:\t Pad \t ActiNum \t Kernel \t Stride \t OutSize \n");
	printf("    CLayer\t %d \t %d \t\t [%d,%d] \t\t [%d,%d] \t\t [%d,%d,%d]\n"
		, cv.PaddingMode, cv.ActiFuncNum, cv.KernelSize[0], cv.KernelSize[1]
		, cv.Stride[0], cv.Stride[1], cv.OutH, cv.OutW, cv.OutC);
	printf("KernelMat[%d,%d]\tWeightMat[%d,%d]\tNablaMat[%d,%d]\n",
		cv.KernelMat.row, cv.KernelMat.col, cv.WeightMat.row, cv.WeightMat.col,
		cv.NablaMat.row, cv.NablaMat.col);
	printf("OriginMat[%d,%d]\n", cv.OriginMat[0].row, cv.OriginMat[0].col);
	printf("SumMat[%d,%d]\tActiMat[%d,%d]\tDeriMat[%d,%d]\tDeltaMat[%d,%d]\n",
		cv.SumMat[0].row, cv.SumMat[0].col, cv.ActiMat[0].row, cv.ActiMat[0].col,
		cv.DeriMat[0].row, cv.DeriMat[0].col, cv.DeltaMat[0].row, cv.DeltaMat[0].col);
	return 0;
}

int DumpDeConvLayer(Layer dec)
{
	printf("Layers:\t Pad \t ActiNum \t Kernel \t Stride \t OutSize \n");
	printf("  deCLayer\t %d \t %d \t\t [%d,%d] \t\t [%d,%d] \t\t [%d,%d,%d]\n"
		, dec.PaddingMode, dec.ActiFuncNum
		, dec.KernelSize[0], dec.KernelSize[1]
		, dec.Stride[0], dec.Stride[1]
		, dec.OutH, dec.OutW, dec.OutC);
	printf("KernelMat[%d,%d]\tNablaMat[%d,%d]\n",
		dec.KernelMat.row, dec.KernelMat.col, dec.NablaMat.row, dec.NablaMat.col);
	printf("OriginMat[%d,%d]\n", dec.OriginMat[0].row, dec.OriginMat[0].col);
	printf("SumMat[%d,%d]\tActiMat[%d,%d]\tDeriMat[%d,%d]\tDeltaMat[%d,%d]\n",
		dec.SumMat[0].row, dec.SumMat[0].col, dec.ActiMat[0].row, dec.ActiMat[0].col,
		dec.DeriMat[0].row, dec.DeriMat[0].col, dec.DeltaMat[0].row, dec.DeltaMat[0].col);
	return 0;
}

int DumpMaxPoolLayer(Layer mp)
{
	printf("Layers:\t Pad \t Kernel \t Stride \t OutSize \n");
	printf("   MPLayer\t %d \t [%d,%d] \t\t [%d,%d] \t\t [%d,%d,%d]\n"
		, mp.PaddingMode, mp.KernelSize[0], mp.KernelSize[1]
		, mp.Stride[0], mp.Stride[1], mp.OutH, mp.OutW, mp.OutC);
	printf("KernelMat[%d,%d]\n",mp.KernelMat.row, mp.KernelMat.col);
	printf("OriginMat[%d,%d]\n", mp.OriginMat[0].row, mp.OriginMat[0].col);
	printf("ActiMat[%d,%d]\tDeltaMat[%d,%d]\n",
		mp.ActiMat[0].row, mp.ActiMat[0].col,mp.DeltaMat[0].row, mp.DeltaMat[0].col);
	return 0;
}

int DumpAvePoolLayer(Layer ap)
{
	printf("Layers:\t Pad \t Kernel \t Stride \t OutSize \n");
	printf("   APLayer\t %d \t [%d,%d] \t\t [%d,%d] \t\t [%d,%d,%d]\n"
		, ap.PaddingMode, ap.KernelSize[0], ap.KernelSize[1]
		, ap.Stride[0], ap.Stride[1], ap.OutH, ap.OutW, ap.OutC);
	printf("KernelMat[%d,%d]\n", ap.KernelMat.row, ap.KernelMat.col);
	printf("OriginMat[%d,%d]\n", ap.OriginMat[0].row, ap.OriginMat[0].col);
	printf("ActiMat[%d,%d]\tDeltaMat[%d,%d]\n",
		ap.ActiMat[0].row, ap.ActiMat[0].col, ap.DeltaMat[0].row, ap.DeltaMat[0].col);
	return 0;
}

int DumpLayersofCNN(CNN cnn)
{
	printf("-----------------------Layers---------------------\n");
	DumpInputLayer(cnn.Layers[0]);
	int i;
	for (i = 1; i < cnn.HiddenLayerNum + 2; i++)
	{
		if (cnn.Layers[i].LayerType == 0)
		{
			printf("%d ", i);
			DumpFullConnLayer(cnn.Layers[i]);
		}
		if (cnn.Layers[i].LayerType == 1)
		{
			printf("%d ", i);
			DumpConvLayer(cnn.Layers[i]);
		}
		if (cnn.Layers[i].LayerType == 2)
		{
			printf("%d ", i);
			DumpDeConvLayer(cnn.Layers[i]);
		}
		if (cnn.Layers[i].LayerType == 3)
		{
			printf("%d ", i);
			DumpMaxPoolLayer(cnn.Layers[i]);
		}
		if (cnn.Layers[i].LayerType == 4)
		{
			printf("%d ", i);
			DumpAvePoolLayer(cnn.Layers[i]);
		}
	}
	printf("--------------------------------------------------\n");
	return 0;
}

int DumpCNN(CNN cnn)
{
	printf("================================================================= CNN Dump ====================================================================\n");
	if (cnn.sampleCapacity == -1) {
		printf("\t\t\tDataSet parameter 'sampleCapacity' uninitialized!!!\n");
		return -1;
	}
	printf("sampleCapacity:\t\t%d\n", cnn.sampleCapacity);

	if (cnn.SampleRow == -1) {
		printf("\t\t\tDataSet parameter 'SampleRow' uninitialized!!!\n");
		return -1;
	}
	printf("SampleRow:\t\t%d\n", cnn.SampleRow);

	if (cnn.SampleCol == -1) {
		printf("\t\t\tDataSet parameter 'SampleCol' uninitialized!!!\n");
		return -1;
	}
	printf("SampleCol:\t\t%d\n", cnn.SampleCol);

	if (cnn.SampleCha == -1) {
		printf("\t\t\tDataSet parameter 'SampleCha' uninitialized!!!\n");
		return -1;
	}
	printf("SampleCha:\t\t%d\n", cnn.SampleCha);

	if (cnn.HiddenLayerNum == -1) {
		printf("\t\t\tDataSet parameter 'HiddenLayerNum' uninitialized!!!\n");
		return -1;
	}
	printf("HiddenLayerNum:\t\t%d\n", cnn.HiddenLayerNum);

	if (cnn.WeightInitWayNum == -1) {
		printf("\t\t\tDataSet parameter 'WeightInitWayNum' uninitialized!!!\n");
		return -1;
	}
	printf("WeightInitWayNum:\t%d\n", cnn.WeightInitWayNum);

	if (cnn.ClassificationNum == -1) {
		printf("\t\t\tDataSet parameter 'ClassificationNum' uninitialized!!!\n");
		return -1;
	}
	printf("ClassificationNum:\t%d\n", cnn.ClassificationNum);

	if (cnn.LossFuncNum == -1) {
		printf("\t\t\tDataSet parameter 'LossFuncNum' uninitialized!!!\n");
		return -1;
	}
	printf("LossFuncNum:\t\t%d\n", cnn.LossFuncNum);

	if (cnn.OnehotMat.element == NULL) {
		printf("\t\t\tDataSet parameter 'OnehotMat' uninitialized!!!\n");
		return -1;
	}
	printf("OnehotMat:\t\t[%d,%d]\n", cnn.OnehotMat.row, cnn.OnehotMat.col);

	if (cnn.Layers == NULL) {
		printf("\t\t\tDataSet parameter 'Layers' uninitialized!!!\n");
		return -1;
	}
	DumpLayersofCNN(cnn);

	printf("============================================================ CNN Dump Finished ================================================================\n");

	return 0;
}

int DumpKWofCNN(CNN cnn)
{
	printf("--------Dump KernelMat and WeightMat of CNN--------\n");
	int i;
	for (i = 1; i < cnn.HiddenLayerNum + 2; ++i)
	{
		printf("%d Layers:\n", i);
		MatDump(&cnn.Layers[i].KernelMat);
		if (cnn.Layers[i].LayerType == 0 || cnn.Layers[i].LayerType == 1)
		{
			MatDump(&cnn.Layers[i].WeightMat);
		}
	}
	printf("----------------------------------------------------\n");
	return 0;
}

int DumpMatplusHWC(Mat* matp, int hgh)
{
	MatplusShape(matp, hgh);
	int i, j, k;
	for (i = 0; i < matp[0].col; ++i)
	{
		printf("Channel %d:\n", i);
		for (j = 0; j < hgh; ++j)
		{
			for (k = 0; k < matp[0].row; ++k)
			{
				printf("%.20f ", (matp[j].element[k])[i]);
			}
			printf("\n");
		}
	}
	return 0;
}


int DumpMatplusHWCo(Mat* mp,int hgh)
{
	int i, j, k;
	printf("\n");
	MatplusShape(mp, hgh);
	for (i = 0; i < hgh;++i)
	{
		printf("Height %d :\n", i);
		for (j = 0; j < mp[0].row;++j)
		{
			for (k = 0; k < mp[0].col;++k)
			{
				printf("%f ", (mp[i].element[j])[k]);
			}
			printf("\n");
		}
	}
	return 0;
}

int DumpMat(Mat mat)
{
	MatShape(&mat);
	int i, j;
	for (i = 0; i < mat.row;++i)
	{
		for(j=0;j<mat.col;++j)
		{
			printf("%f ", (mat.element[i])[j]);
		}
		printf("\n");
	}
	return 0;
}

/************************************************************************/
/*                             dump����                                  */
/************************************************************************/

/************************************************************************/
/*                              CNN����                                  */
/************************************************************************/

Layer* SpaceCreateLayers(CNN *cnn)
{
	int i;
	cnn->Layers = (Layer*)malloc((cnn->HiddenLayerNum + 2) * sizeof(Layer));
	for (i = 0; i < cnn->HiddenLayerNum + 2; i++)
	{
		InitLayer(&cnn->Layers[i]);
	}
	return cnn->Layers;
}


Layer* SpaceCreateInputLayer(Layer *input)
{
	int i;
	input->ActiMat = (Mat*)malloc(input->OutH * sizeof(Mat));
	for (i = 0; i < input->OutH; ++i)
	{
		MatCreate(&input->ActiMat[i], input->OutW, input->OutC);
	}
	return input;
}

Layer* SpaceCreateFullConnLayer(Layer *fc, int InH, int InW, int InC)
{
	int InDim = InH * InW * InC;
	fc->OriginMat = (Mat*)malloc(sizeof(Mat));
	MatCreate(fc->OriginMat, 1, InDim + 1);
	MatCreate(&fc->KernelMat, InDim, fc->OutC);
	MatCreate(&fc->WeightMat, InDim + 1, fc->OutC);
	MatCreate(&fc->NablaMat, InDim + 1, fc->OutC);
	fc->SumMat = (Mat*)malloc(sizeof(Mat));
	MatCreate(fc->SumMat, 1, fc->OutC);
	fc->ActiMat = (Mat*)malloc(sizeof(Mat));
	MatCreate(fc->ActiMat, 1, fc->OutC);
	fc->DeriMat = (Mat*)malloc(sizeof(Mat));
	MatCreate(fc->DeriMat, 1, fc->OutC);
	fc->DeltaMat = (Mat*)malloc(sizeof(Mat));
	MatCreate(fc->DeltaMat, 1, fc->OutC);
	return fc;
}

Layer* SpaceCreateConvLayer(Layer *cv, int InH, int InW, int InC)
{
	int i;
	MatCreate(&cv->KernelMat, cv->KernelSize[0] * cv->KernelSize[1] * InC, cv->OutC);
	MatCreate(&cv->WeightMat, cv->KernelSize[0] * cv->KernelSize[1] * InC + 1, cv->OutC);
	MatCreate(&cv->NablaMat, cv->KernelSize[0] * cv->KernelSize[1] * InC + 1, cv->OutC);
	cv->OriginMat = (Mat*)malloc(cv->OutH * sizeof(Mat));
	cv->SumMat = (Mat*)malloc(cv->OutH * sizeof(Mat));
	cv->ActiMat = (Mat*)malloc(cv->OutH * sizeof(Mat));
	cv->DeriMat = (Mat*)malloc(cv->OutH * sizeof(Mat));
	cv->DeltaMat = (Mat*)malloc(cv->OutH * sizeof(Mat));
	for (i = 0; i < cv->OutH; ++i)
	{
		MatCreate(&cv->OriginMat[i], cv->OutW, cv->KernelSize[0] * cv->KernelSize[1] * InC + 1);
		MatCreate(&cv->SumMat[i], cv->OutW, cv->OutC);
		MatCreate(&cv->ActiMat[i], cv->OutW, cv->OutC);
		MatCreate(&cv->DeriMat[i], cv->OutW, cv->OutC);
		MatCreate(&cv->DeltaMat[i], cv->OutW, cv->OutC);

	}
	return cv;
}

Layer* SpaceCreateDeConvLayer(Layer *dec, int InH, int InW, int InC)
{
	int i;
	MatCreate(&dec->KernelMat, InC, dec->KernelSize[0] * dec->KernelSize[1] * dec->OutC);
	MatCreate(&dec->NablaMat, InC, dec->KernelSize[0] * dec->KernelSize[1] * dec->OutC);
	dec->OriginMat = (Mat*)malloc(dec->OutH * sizeof(Mat));
	dec->SumMat = (Mat*)malloc(dec->OutH * sizeof(Mat));
	dec->ActiMat = (Mat*)malloc(dec->OutH * sizeof(Mat));
	dec->DeriMat = (Mat*)malloc(dec->OutH * sizeof(Mat));
	dec->DeltaMat = (Mat*)malloc(dec->OutH * sizeof(Mat));
	for (i = 0; i < dec->OutH; ++i)
	{
		MatCreate(&dec->OriginMat[i], InW, dec->KernelSize[0] * dec->KernelSize[1] * dec->OutC);
		MatCreate(&dec->SumMat[i], dec->OutW, dec->OutC);
		MatCreate(&dec->ActiMat[i], dec->OutW, dec->OutC);
		MatCreate(&dec->DeriMat[i], dec->OutW, dec->OutC);
		MatCreate(&dec->DeltaMat[i], dec->OutW, dec->OutC);
	}
	return dec;
}

Layer* SpaceCreateMaxPoolLayer(Layer *mp, int InH, int InW, int InC)
{
	int i;
	MatCreate(&mp->KernelMat, mp->OutH * mp->OutW, mp->OutC);
	mp->OriginMat = (Mat*)malloc(mp->OutH * sizeof(Mat));
	mp->ActiMat = (Mat*)malloc(mp->OutH * sizeof(Mat));
	mp->DeltaMat = (Mat*)malloc(mp->OutH * sizeof(Mat));
	for (i = 0; i < mp->OutH; ++i)
	{
		MatCreate(&mp->OriginMat[i], mp->OutW, mp->KernelSize[0] * mp->KernelSize[1] * InC);
		MatCreate(&mp->ActiMat[i], mp->OutW, mp->OutC);
		MatCreate(&mp->DeltaMat[i], mp->OutW, mp->OutC);
	}
	return mp;
}

Layer* SpaceCreateAvePoolLayer(Layer *ap, int InH, int InW, int InC)
{
	int i;
	MatCreate(&ap->KernelMat, ap->KernelSize[0] * ap->KernelSize[1] * InC, ap->OutC);
	ap->OriginMat = (Mat*)malloc(ap->OutH * sizeof(Mat));
	ap->ActiMat = (Mat*)malloc(ap->OutH * sizeof(Mat));
	ap->DeltaMat = (Mat*)malloc(ap->OutH * sizeof(Mat));
	for (i = 0; i < ap->OutH; ++i)
	{
		MatCreate(&ap->OriginMat[i], ap->OutW, ap->KernelSize[0] * ap->KernelSize[1] * InC);
		MatCreate(&ap->ActiMat[i], ap->OutW, ap->OutC);
		MatCreate(&ap->DeltaMat[i], ap->OutW, ap->OutC);
	}
	return ap;
}

int CreateNNMatSpace(CNN *cnn)
{
	int i;
	SpaceCreateInputLayer(&cnn->Layers[0]);
	for (i = 1; i < cnn->HiddenLayerNum + 2; ++i)
	{
		if (cnn->Layers[i].LayerType == 0)
		{
			SpaceCreateFullConnLayer(&cnn->Layers[i], cnn->Layers[i - 1].OutH, cnn->Layers[i - 1].OutW, cnn->Layers[i - 1].OutC);
			continue;
		}
		if (cnn->Layers[i].LayerType == 1)
		{
			SpaceCreateConvLayer(&cnn->Layers[i], cnn->Layers[i - 1].OutH, cnn->Layers[i - 1].OutW, cnn->Layers[i - 1].OutC);
			continue;
		}
		if (cnn->Layers[i].LayerType == 2)
		{
			SpaceCreateDeConvLayer(&cnn->Layers[i], cnn->Layers[i - 1].OutH, cnn->Layers[i - 1].OutW, cnn->Layers[i - 1].OutC);
			continue;
		}
		if (cnn->Layers[i].LayerType == 3)
		{
			SpaceCreateMaxPoolLayer(&cnn->Layers[i], cnn->Layers[i - 1].OutH, cnn->Layers[i - 1].OutW, cnn->Layers[i - 1].OutC);
			continue;
		}
		if (cnn->Layers[i].LayerType == 4)
		{
			SpaceCreateAvePoolLayer(&cnn->Layers[i], cnn->Layers[i - 1].OutH, cnn->Layers[i - 1].OutW, cnn->Layers[i - 1].OutC);
			continue;
		}
	}
	return 0;
}

/************************************************************************/
/*                              CNN����                                  */
/************************************************************************/

/************************************************************************/
/*                           Ȩֵ��ʼ����������                         */
/************************************************************************/
float gaussrand_NORMAL() {
	float V1 = 0., V2 = 0., S = 0.;
	int phase = 0;
	int count = 0;
	float X;


	if (phase == 0) {
		while (count == 0 || (S >= 1 || S == 0)) {
			float U1 = (float)(rand() % 10000) / 10000.f;
			float U2 = (float)(rand() % 10000) / 10000.f;


			V1 = 2 * U1 - 1;
			V2 = 2 * U2 - 1;
			S = V1 * V1 + V2 * V2;
			count += 1;
		};

		float temp_S_1 = (float)log(S);
		X = V1 * (float)sqrt(-2 * temp_S_1 / S);
	}
	else
	{
		float temp_S_2 = (float)log(S);
		X = V2 * (float)sqrt(-2 * temp_S_2 / S);
	}



	phase = 1 - phase;


	return X;
}

float gaussrand(float mean, float stdc) {
	return mean + gaussrand_NORMAL() * stdc;
}

// ȫ0��ʼ��
Mat* MatInitZero(Mat *src)
{
	MatZeros(src);
	return src;
}

/* ��ʼ����0��ֵ 0.01�����ı�׼������*/
Mat* MatInitRandomNormalization(Mat *src)
{
	srand((unsigned int)time(NULL));  // set randon seed

	int row, col;

	//weight
	for (row = 0; row < src->row; ++row) {
		for (col = 0; col < src->col; ++col) {
			(src->element[row])[col] = gaussrand(0.f, 0.1f);  // mean stdc
		}
	}
	return src;
}

Mat* MatInitXavier(Mat *src)
{
	srand((unsigned int)time(NULL));  // set randon seed

	int row, col;
	//weight
	for (row = 0; row < src->row; ++row) {
		for (col = 0; col < src->col; ++col) {
			(src->element[row])[col] = gaussrand(0.f, 0.1f) * (float)sqrt(1.f / src->row);  // mean stdc
		}
	}
	//set bias row 0
	//for (col = 0; col < src->col; ++col){
	//	(src->element[0])[col] = 0.f;
	//}
	return src;
}

Mat* MatInitHe(Mat *src)
{
	//srand((unsigned int)time(NULL));  // set randon seed
	srand(19950826);

	int row, col;
	//weight
	for (row = 0; row < src->row; ++row) {
		for (col = 0; col < src->col; ++col) {
			(src->element[row])[col] = gaussrand(0.f, 0.9f) * (float)sqrt(2.f / (src->row));  // mean stdc
		}
	}
	//set bias row 0
	//for (col = 0; col < src->col; ++col){
	//	(src->element[0])[col] = 0.f;
	//}
	return src;
}
/************************************************************************/
/*                           Ȩֵ��ʼ����������                         */
/************************************************************************/


/************************************************************************/
/*                             CNN��ʼ��                                 */
/************************************************************************/
void WeightInit_ChooseWay(Mat *Weight, int Style_initWeight) {
	if (Style_initWeight == 0) {
		MatInitZero(Weight);
	}
	else if (Style_initWeight == 1) {
		MatInitRandomNormalization(Weight);
	}
	else if (Style_initWeight == 2) {
		MatInitXavier(Weight);
	}
	else if (Style_initWeight == 3) {
		MatInitHe(Weight);
	}
	else {
		printf("error for WeightInit_ChooseWay, please check Style_initWeight variable!\n");
	}

}

void WeightInit_AverPool(Mat* KernelMat, int kers[2])
{
	int area = kers[0] * kers[1];
	int row, col;
	for (row = 0; row < KernelMat->row; ++row)
	{
		for (col = 0; col < KernelMat->col; ++col)
		{
			(KernelMat->element[row])[col] = 1.0 / (float)area;
		}
	}
}

int NNWeightInit(CNN *cnn)
{
	int i;
	for (i = 1; i < cnn->HiddenLayerNum + 2; ++i)
	{
		if (cnn->Layers[i].LayerType == 0 || cnn->Layers[i].LayerType == 1)
		{
			WeightInit_ChooseWay(&cnn->Layers[i].WeightMat, cnn->WeightInitWayNum);
			MatReduceRow(&cnn->Layers[i].WeightMat, &cnn->Layers[i].KernelMat);
		}
		else if (cnn->Layers[i].LayerType == 4)
		{
			WeightInit_AverPool(&cnn->Layers[i].KernelMat, cnn->Layers[i].KernelSize);
		}
		else{
			WeightInit_ChooseWay(&cnn->Layers[i].KernelMat, cnn->WeightInitWayNum);
		}
	}
	return 0;
}

/************************************************************************/
/*                             CNN��ʼ��                                 */
/************************************************************************/

/************************************************************************/
/*                           ���������                               */
/************************************************************************/
float sigmoid(float z)
{
	return (float)(1 / (1 + exp(-z)));
}

//ֱ�ӵ��� math.h �ﺯ��
//float tanh(float z){
//	return (exp(z) - exp(-z)) / (exp(z) + exp(-z));
//}
//

float relu(float z)
{
	return z > 0 ? z : 0;
}

float leakyRelu(float z, float a)
{
	return z < 0 ? a : z;
}

Mat* MatSoftmax(Mat *src, Mat *dst)
{
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatSofmax\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	Mat tempV;
	MatCreate(&tempV, src->row, 1);   //����ʱ��������ֵ�Լ�������ֵ����

	MatRowMax(src, &tempV);   //��������ֵ����

	MatVectorSub(src, &tempV, dst);   //������������

	MatExp(dst, dst);   //����������

	MatRowSum(dst, &tempV);   //������������

	MatVectorDiv(dst, &tempV, dst);   //������������

	MatDelete(&tempV);

	return dst;
}

Mat* MatNoneActi(Mat *src, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatNoneActi\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = (src->element[row])[col];
	}

	return dst;
}

Mat* MatSigmoid(Mat *src, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatSigmoid\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = sigmoid((src->element[row])[col]);
	}

	return dst;
}

Mat* MatTanh(Mat *src, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatTanh\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = (float)tanh((src->element[row])[col]);
	}

	return dst;
}

Mat* MatRelu(Mat *src, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatRelu\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = relu((src->element[row])[col]);
	}

	return dst;
}

Mat* MatLeakyRelu(float a, Mat *src, Mat *dst)
{
	int row, col;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatLeakyRelu\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = leakyRelu((src->element[row])[col], a);
	}

	return dst;
}

//Derivative
Mat* MatDerivationSoftmax(Mat *src, Mat *dst)
{
	int row, col, i;
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatDerivationSofmax\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	MatSoftmax(src, src);
	MatZeros(dst);

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			for (i = 0; i < src->col; i++) {
				if (i == col) {
					(dst->element[row])[col] += (src->element[row])[i] * (1 - (src->element[row])[col]);
				}
				else {
					(dst->element[row])[col] += -(src->element[row])[i] * (src->element[row])[col];
				}
				//printf("%f\n", (dst->element[row])[col]);
			}
			//printf("sum=%f\n", (dst->element[row])[col]);
		}
	}


	return dst;
}

Mat* MatDerivationNoneActi(Mat *src, Mat *dst)
{	//�������ֵΪ1
	int row, col;
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatDerivationNoneActi\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			(dst->element[row])[col] = 1.f;
	}

	return dst;
}

Mat* MatDerivationSigmoid(Mat *src, Mat *dst)
{
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatDerivationSigmoid\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif
	Mat temp1Mat;
	Mat temp2Mat;
	MatCreate(&temp1Mat, src->row, src->col);
	MatCreate(&temp2Mat, src->row, src->col);

	MatSigmoid(src, &temp1Mat);
	//MatDump(&temp1Mat);
	MatNumMul(-1.f, &temp1Mat, &temp2Mat);
	//MatDump(&temp2Mat);
	MatNumAdd(1.f, &temp2Mat, &temp2Mat);
	//MatDump(&temp2Mat);
	MatProduct(&temp1Mat, &temp2Mat, dst);
	//MatDump(dst);
	MatDelete(&temp1Mat);
	MatDelete(&temp2Mat);
	return dst;
}

Mat* MatDerivationTanh(Mat *src, Mat *dst)
{
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatDerivationTanh\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	Mat tempMat;
	MatCreate(&tempMat, src->row, src->col);

	MatTanh(src, &tempMat);
	//MatDump(&tempMat);
	MatSquare(&tempMat, &tempMat);
	//MatDump(&tempMat);
	MatNumMul(-1.f, &tempMat, &tempMat);
	//MatDump(&tempMat);
	MatNumAdd(1.f, &tempMat, dst);

	MatDelete(&tempMat);
	return dst;
}

Mat* MatDerivationRelu(Mat *src, Mat *dst)
{
	int row, col;
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatDerivationRelu\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif
	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			if ((src->element[row])[col]>0) {
				(dst->element[row])[col] = 1.f;
			}
			else {
				(dst->element[row])[col] = 0.f;
			}
		}

	}

	return dst;
}

Mat* MatDerivationLeakyRelu(float a, Mat *src, Mat *dst)
{
	int row, col;
#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for MatDerivationLeakyRelu\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif
	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++) {
			if ((src->element[row])[col]>0) {
				(dst->element[row])[col] = 1.f;
			}
			else {
				(dst->element[row])[col] = a;
			}
		}

	}

	return dst;
}
/************************************************************************/
/*                           ���������                               */
/************************************************************************/

/************************************************************************/
/*                           ��ʧ��������                               */
/************************************************************************/
Mat* OneHot(Mat *src, int k, Mat *dst)
{
	int row;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != 1 || dst->col != k) {
		printf("\t\terr check, unmathed matrix for Onehot\t\t\n");
		printf("\t\tsrcMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tThe number of class:\n\t\t\t");
		printf("%d\n", k);
		printf("\t\tdstMatShape:\n\t\t\t");
		MatShape(dst);
		return NULL;
	}
#endif

	MatZeros(dst);

	for (row = 0; row < dst->row; row++) {
		(dst->element[row])[int((src->element[row])[0])] = 1.0f;
	}

	return dst;
}

/*��������*/
float MSE(Mat *src, Mat *dst)
{
	int row;
	float loss = 0.f;
	Mat sub_square_mat;
	Mat sum_row_mat;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for Loss Function MSE\t\t\n");
		printf("\t\tPredictoinMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tOneHotMatShape:\n\t\t\t");
		MatShape(dst);
		return -1.;     // �������鲻���ط��� -1��
	}
#endif

	MatCreate(&sub_square_mat, src->row, src->col);
	MatCreate(&sum_row_mat, src->row, 1);

	MatSub(src, dst, &sub_square_mat);
	//MatDump(&sub_square_mat);
	MatSquare(&sub_square_mat, &sub_square_mat);
	//MatDump(&sub_square_mat);
	MatRowSum(&sub_square_mat, &sum_row_mat);
	//MatDump(&sum_row_mat);

	for (row = 0; row < src->row; ++row) {
		//printf("%f\t", (sum_row_mat.element[row])[0]);
		loss = loss + (sum_row_mat.element[row])[0];
	}
	loss = loss / float(src->row);

	MatDelete(&sub_square_mat);
	MatDelete(&sum_row_mat);
	return loss;
}


float CrossEntropy(Mat *src, Mat *dst)
{
	int row, col;
	float loss = 0.f;

#ifdef MAT_LEGAL_CHECKING
	if (src->row != dst->row || src->col != dst->col) {
		printf("\t\terr check, unmathed matrix for Loss Function CrossEntropy\t\t\n");
		printf("\t\tPredictoinMatShape:\n\t\t\t");
		MatShape(src);
		printf("\t\tOneHotMatShape:\n\t\t\t");
		MatShape(dst);
		return -1.;     // �������鲻���ط��� -1��
	}
#endif

	for (row = 0; row < src->row; row++) {
		for (col = 0; col < src->col; col++)
			loss += (float)(-1 * (dst->element[row])[col] * log((src->element[row])[col]));
	}

	loss = loss / (src->row);

	return loss;
}


//�Լ���ֵ������
Mat * MSEDerivative(Mat *ActiMat, Mat *DerivativeActiMat, Mat One_hotMat) {

#ifdef MAT_LEGAL_CHECKING
	if (ActiMat->row != DerivativeActiMat->row || ActiMat->col != DerivativeActiMat->col || ActiMat->row != One_hotMat.row || ActiMat->col != One_hotMat.col) {
		printf("\t\terr check, unmathed matrix for Loss Function MSEDerivative\t\t\n");
		printf("\t\tActiMatShape:\n\t\t\t");
		MatShape(ActiMat);
		printf("\t\tDerivativeActiMatShape:\n\t\t\t");
		MatShape(DerivativeActiMat);
		printf("\t\tOne_hotMatShape:\n\t\t\t");
		MatShape(&One_hotMat);
		return NULL;     // �������鲻���ط��� -1��
	}
#endif

	//partial L /partial z = z - y
	return MatSub(ActiMat, &One_hotMat, DerivativeActiMat);

}


Mat * CrossEntropyDerivative(Mat *ActiMat, Mat *DerivativeActiMat, Mat One_hotMat) {
	int row, col;
#ifdef MAT_LEGAL_CHECKING
	if (ActiMat->row != DerivativeActiMat->row || ActiMat->col != DerivativeActiMat->col || ActiMat->row != One_hotMat.row || ActiMat->col != One_hotMat.col) {
		printf("\t\terr check, unmathed matrix for Loss Function CrossEntropyDerivative\t\t\n");
		printf("\t\tActiMatShape:\n\t\t\t");
		MatShape(ActiMat);
		printf("\t\tDerivativeActiMatShape:\n\t\t\t");
		MatShape(DerivativeActiMat);
		printf("\t\tOne_hotMatShape:\n\t\t\t");
		MatShape(&One_hotMat);
		return NULL;     // �������鲻���ط��� -1��
	}
#endif

	//partial L /partial z = -(y / z)
	for (row = 0; row < ActiMat->row; row++) {
		for (col = 0; col < ActiMat->col; col++)
			//div 0 error
			if (equal((ActiMat->element[row])[col], 0.f) == 1) {
				(DerivativeActiMat->element[row])[col] = -(One_hotMat.element[row])[col] * 10000000000;
			}
			else {
				(DerivativeActiMat->element[row])[col] = -(One_hotMat.element[row])[col] / (ActiMat->element[row])[col];
			}
	}

	return DerivativeActiMat;

}
/************************************************************************/
/*                           ��ʧ��������                               */
/************************************************************************/

/************************************************************************/
/*                             ������������                               */
/************************************************************************/

Mat* MatPadReConstructBias(Mat *des, int des_h, Mat *src, int src_h, int kers[2], int strd[2], int pad)
{
	// RC&P will be difficult in verified dimension ,so verify is maken sure by Space Create
	int pdh, pdw;
	if (pad == 0)
	{
		pdh = 0;
		pdw = 0;
	}
	else if(pad == 1)
	{
		if (strd[0] == 1) pdh = kers[0] - 1;
		else pdh = kers[0] - src_h%strd[0];
		if (strd[1] == 1) pdw = kers[1] - 1;
		else pdw = kers[1] - src[0].row%strd[1];
	}
	else {
		return NULL;
	}
	int i, j, k, m, n;
	for (i = 0; i < des_h; ++i)
	{
		for (j = 0; j < des[0].row; ++j)
		{
			(des[i].element[j])[0] = 1.0;
			for (k = 0; k < src[i].col; ++k)
			{
				for (m = 0; m < kers[0]; ++m)
				{
					for (n = 0; n < kers[1]; ++n)
					{
						if (i*strd[0] + m < pdh / 2 ||
							j*strd[1] + n < pdw / 2 ||
							i*strd[0] + m >= src_h + pdh / 2 ||
							j*strd[1] + n >= src[0].row + pdw / 2)
						{
							(des[i].element[j])[k*kers[0] * kers[1] + m*kers[1] + n + 1] = 0;
						}
						else {
							(des[i].element[j])[k*kers[0] * kers[1] + m*kers[1] + n + 1]
								= (src[i*strd[0] + m - pdh / 2].element[j*strd[1] + n - pdw / 2])[k];

						}
					}
				}
			}
		}
	}
	return des;
}

Mat* MatPadReConstruct(Mat *des, int des_h, Mat *src, int src_h, int kers[2], int strd[2], int pad)
{
	int pdh, pdw;
	if (pad == 0)
	{
		pdh = 0;
		pdw = 0;
	}
	else if (pad == 1)
	{
		if (strd[0] == 1) pdh = kers[0] - 1;
		else pdh = kers[0] - src_h%strd[0];
		if (strd[1] == 1) pdw = kers[1] - 1;
		else pdw = kers[1] - src[0].row%strd[1];
	}
	else {
		return NULL;
	}
	int i, j, k, m, n;
	for (i = 0; i < des_h; ++i)
	{
		for (j = 0; j < des[0].row; ++j)
		{
			for (k = 0; k < src[i].col; ++k)
			{
				for (m = 0; m < kers[0]; ++m)
				{
					for (n = 0; n < kers[1]; ++n)
					{
						if (i*strd[0] + m < pdh / 2 ||
							j*strd[1] + n < pdw / 2 ||
							i*strd[0] + m >= src_h + pdh / 2 ||
							j*strd[1] + n >= src[0].row + pdw / 2)
						{
							(des[i].element[j])[k*kers[0] * kers[1] + m*kers[1] + n] = 0;
						}
						else {
							(des[i].element[j])[k*kers[0] * kers[1] + m*kers[1] + n]
								= (src[i*strd[0] + m - pdh / 2].element[j*strd[1] + n - pdw / 2])[k];

						}
					}
				}
			}
		}
	}
	return des;
}

Mat* MatplusMulMat(Mat *sm, int sm_h, Mat *om, int om_h, Mat wm)
{
	if (om_h!=sm_h||om[0].row!=sm[0].row||wm.row!=om[0].col||wm.col!=sm[0].col) {
		printf("\t\terr check, unmathed matrix for MatplusMulMat\t\t\n");
		printf("\t\tOriginMatShape:\n\t\t\t");
		MatplusShape(om, sm_h);
		printf("\t\tWeightMatShape:\n\t\t\t");
		MatShape(&wm);
		printf("\t\tSumMatShape:\n\t\t\t");
		MatplusShape(sm, sm_h);
		return NULL;     // �������鲻���ط��� -1��
	}
	int i, j, k, l;

	for (i = 0; i < sm_h; ++i)
	{
		for (j = 0; j < sm[0].row; ++j)
		{
			for (k = 0; k < sm[0].col; ++k)
			{
				float tmp = 0.0;
				for (l = 0; l < om[0].col; ++l)
				{
					tmp = tmp + (om[i].element[j])[l] * (wm.element[l])[k];
				}
				(sm[i].element[j])[k] = tmp;
			}
		}
	}
	return sm;
}

Mat* MatplusMulMat_T_unmatch(Mat *dst, int dst_h, Mat *src, int src_h, Mat km)
{
	if (src_h*src[0].row*km.row!=dst_h*dst[0].row*dst[0].col||src[0].col!=km.col) {
		printf("\t\terr check, unmathed matrix for MatplusMulMat_T_unmatch\t\t\n");
		printf("\t\tDstShape:\n\t\t\t");
		MatplusShape(dst, dst_h);
		printf("\t\tKernelMatShape:\n\t\t\t");
		MatShape(&km);
		printf("\t\tSrcShape:\n\t\t\t");
		MatplusShape(src, src_h);
		return NULL;     // �������鲻���ط��� -1��
	}
	int cnt1 = 0, cnt2 = 0, cnt3 = 0;
	int i, j, k, l;
	for (i = 0; i < src_h; ++i)
	{
		for (j = 0; j < src[0].row; ++j)
		{
			for (k = 0; k < km.row; ++k)
			{
				float tmp = 0.0;
				for (l = 0; l < km.col; ++l)
				{
					tmp = tmp + (src[i].element[j])[l] * (km.element[k])[l];
				}
				(dst[cnt1].element[cnt2])[cnt3] = tmp;
				++cnt3;
				if (cnt3 >= dst[0].col) {
					++cnt2;
					cnt3 = 0;
					if (cnt2 >= dst[0].row) {
						++cnt1;
						cnt2 = 0;
					}
				}
			}
		}
	}
}

Mat* MatplusMulMat_T(Mat *dst, int dst_h, Mat *src, int src_h, Mat km)
{
	if (src_h != dst_h || src[0].row != dst[0].row || km.row != dst[0].col || src[0].col != km.col) {
		printf("\t\terr check, unmathed matrix for MatplusMulMat_T\t\t\n");
		printf("\t\tDstShape:\n\t\t\t");
		MatplusShape(dst, dst_h);
		printf("\t\tKernelMatShape:\n\t\t\t");
		MatShape(&km);
		printf("\t\tSrcShape:\n\t\t\t");
		MatplusShape(src, src_h);
		return NULL;     // �������鲻���ط��� -1��
	}
	int i, j, k, l;

	for (i = 0; i < dst_h; ++i)
	{
		for (j = 0; j < dst[0].row; ++j)
		{
			for (k = 0; k < dst[0].col; ++k)
			{
				float tmp = 0.0;
				for (l = 0; l < src[0].col; ++l)
				{
					tmp = tmp + (src[i].element[j])[l] * (km.element[k])[l];
				}
				(dst[i].element[j])[k] = tmp;
			}
		}
	}
	return dst;
}

Mat* MatDeConstruct(Mat *des, int des_h, Mat *src, int src_h, int kers[2], int strd[2])
{
	if(des_h!=(src_h-1)*strd[0]+kers[0]||des[0].row!=(src[0].row-1)*strd[1]+kers[1]||kers[0]*kers[1]*des[0].col!=src[0].col)
	{
		printf("\t\terr check, unmathed matrix for MatDeConstruct\t\t\n");
		printf("\t\tDstShape:\n\t\t\t");
		MatplusShape(des, des_h);
		printf("\t\tSrcShape:\n\t\t\t");
		MatplusShape(src, src_h);
		return NULL;     // �������鲻���ط��� -1��
	}
	MatplusZeros(des, des_h);
	int i, j, k;
	for (i = 0; i < src_h; ++i)
	{
		for (j = 0; j < src[0].row; ++j)
		{
			for (k = 0; k < src[i].col; ++k)
			{
				int d1, d2, d3;
				d1 = i*strd[0] + (k / kers[1]) % kers[0]; // (k % (kers[0] * kers[1])) / kers[1]
				d2 = j*strd[1] + k%kers[1]; // (k % (kers[0] * kers[1])) % kers[1]
				d3 = k / (kers[0] * kers[1]);
				(des[d1].element[d2])[d3] = (des[d1].element[d2])[d3] + (src[i].element[j])[k];
			}
		}
	}
	return des;
}

// des<2d> = flat12(src1<3d>)^T * flat12(src2<3d>)
Mat Matplus_TMulMatplus(Mat *des, Mat* src1, int src1_h, Mat *src2, int src2_h)
{
#ifdef MAT_LEGAL_CHECKING
	if (src1_h != src2_h || src1[0].row != src2[0].row || des->row != src1[0].col || des->col != src2[0].col) {
		printf("\t\terr check, unmathed matrix for Matplus_TMulMatplus\t\t\n");
		/*printf("\t\tActiMatShape:\n\t\t\t");
		MatShape(ActiMat);
		printf("\t\tDerivativeActiMatShape:\n\t\t\t");
		MatShape(DerivativeActiMat);
		printf("\t\tOne_hotMatShape:\n\t\t\t");
		MatShape(&One_hotMat);*/
		return NULL;     // �������鲻���ط��� -1��
	}
#endif
	int i, j, m, n;
	float tmp = 0.0;
	for (i = 0; i < des->row; ++i)
	{
		for (j = 0; j < des->col; ++j)
		{
			for (m = 0; m < src1_h; ++m)
			{
				for (n = 0; n < src1[0].row; ++n)
				{
					tmp = tmp + (src1[m].element[n])[i] * (src1[m].element[n])[j];
				}
			}
			(des->element[i])[j] = tmp;
			tmp = 0.0;
		}
	}
	return *des;
}

Mat Matplus_TMulMatplusSumDes(Mat *des, Mat* src1, int src1_h, Mat *src2, int src2_h, int BatchSize)
{
#ifdef MAT_LEGAL_CHECKING
	if (src1_h != src2_h || src1[0].row != src2[0].row || des->row != src1[0].col || des->col != src2[0].col) {
		printf("\t\terr check, unmathed matrix for Matplus_TMulMatplus\t\t\n");
		/*printf("\t\tActiMatShape:\n\t\t\t");
		MatShape(ActiMat);
		printf("\t\tDerivativeActiMatShape:\n\t\t\t");
		MatShape(DerivativeActiMat);
		printf("\t\tOne_hotMatShape:\n\t\t\t");
		MatShape(&One_hotMat);*/
		return NULL;     // �������鲻���ط��� -1��
	}
#endif
	int i, j, m, n;
	float tmp = 0.0;
	for (i = 0; i < des->row; ++i)
	{
		for (j = 0; j < des->col; ++j)
		{
			for (m = 0; m < src1_h; ++m)
			{
				for (n = 0; n < src1[0].row; ++n)
				{
					tmp = tmp + (src1[m].element[n])[i] * (src1[m].element[n])[j];
				}
			}
			(des->element[i])[j] = (des->element[i])[j] + tmp/BatchSize;
			tmp = 0.0;
		}
	}
	return *des;
}

Mat* MaxChoose(Mat *am, int am_h, Mat *km, Mat* om, int om_h)
{
	if (am_h != om_h || am[0].row != om[0].row) {
		printf("\t\terr check, unmathed matrix for MaxChoose\t\t\n");
		printf("\t\tActiMatShape:\n\t\t\t");
		MatplusShape(am, am_h);
		printf("\t\tOriginActiMatShape:\n\t\t\t");
		MatplusShape(om, om_h);
		printf("\t\tKernelMatShape:\n\t\t\t");
		MatShape(km);
		return NULL;     // �������鲻���ط��� -1��
	}
	int i, j, k, l;
	for (i = 0; i < om_h; ++i)
	{
		for (j = 0; j < om[0].row; ++j)
		{
			for (k = 0; k < km->col; ++k)
			{
				(am[i].element[j])[k] = (om[i].element[j])[k*km->row];
				(km->element[i*km->col + j])[k] = 0;
				for (l = 1; l < km->row; ++l)
				{
					if ((am[i].element[j])[k] < (om[i].element[j])[k*km->row + l])
					{
						(am[i].element[j])[k] = (om[i].element[j])[k*km->row + l];
						(km->element[i*km->col + j])[k] = l;
					}
				}
			}
		}
	}
	return am;
}

Mat* DeMaxChoose(Mat* dm, int dm_h, Mat* km, Mat* dml, int dml_h)
{
#ifdef MAT_LEGAL_CHECKING
	if (dm_h != dml_h || dm[0].row != dml[0].row) {
		printf("\t\terr check, unmathed matrix for DeMaxChoose\t\t\n");
		printf("\t\tDeltaMatShape:\n\t\t\t");
		MatplusShape(dm, dm_h);
		printf("\t\tLaterDeltaShape:\n\t\t\t");
		MatplusShape(dml, dml_h);
		printf("\t\tKernelShape:\n\t\t\t");
		MatShape(km);
		return NULL;     // �������鲻���ط��� -1��
	}
#endif
	int i, j, k, l;
	for (i = 0; i < dm_h; ++i)
	{
		for (j = 0; j < dm[0].row; ++j)
		{
			for (k = 0; k < km->col; ++k)
			{
				for (l = 0; l < km->row; ++l)
				{
					if ((km->element[i*km->col + j])[k] == l)
					{
						(dm[i].element[j])[k*km->row + l] = (dml[i].element[j])[k];
					}
					else {
						(dm[i].element[j])[k*km->row + l] = 0;
					}
				}
			}
		}
	}
	return dm;
}

Mat* FCload(Mat* om, Mat* am, int am_h)
{
#ifdef MAT_LEGAL_CHECKING
	if (om[0].col != am_h * am[0].row * am[0].col) {
		printf("\t\terr check, unmathed matrix for FCload\t\t\n");
		printf("\t\tActiMatShape:\n\t\t\t");
		MatplusShape(am, am_h);
		printf("\t\tOriginMatShape:\n\t\t\t");
		MatplusShape(om, am_h);
		return NULL;     // �������鲻���ط��� -1��
	}
#endif
	int i, j, k;
	int omcnt = 1;
	(om[0].element[0])[0] = 1.0;
	for (int i = 0; i < am_h; ++i)
	{
		for (int j = 0; j < am[0].row; ++j)
		{
			for (int k = 0; k < am[0].col; ++k)
			{
				(om[0].element[0])[omcnt] = (am[i].element[j])[k];
				++omcnt;
			}
		}
	}
	return om;
}

Mat* MatplusActivate(Mat *src, Mat *dst, int hgt, int way)
{
	int i;
	if (way == 0) {
		for (i = 0; i < hgt; ++i)
		{
			MatNoneActi(&src[i], &dst[i]);
		}

	}
	else if (way == 1) {
		for (i = 0; i < hgt; ++i)
		{
			MatSigmoid(&src[i], &dst[i]);
		}
	}
	else if (way == 2) {
		for (i = 0; i < hgt; ++i)
		{
			MatTanh(&src[i], &dst[i]);
		}
	}
	else if (way == 3) {
		for (i = 0; i < hgt; ++i)
		{
			MatRelu(&src[i], &dst[i]);
		}
	}
	else if (way == 4) {
		for (i = 0; i < hgt; ++i)
		{
			MatLeakyRelu(0.2f, &src[i], &dst[i]); //leak = 0.2
		}
	}
	else if (way == 5) {
		for (i = 0; i < hgt; ++i)
		{
			MatSoftmax(&src[i], &dst[i]);
		}
	}
	else {
		printf("error for MatActivate, please check ActiFsHidden  variable!\n");
	}
	return dst;
}

Mat* MatplusDerivate(Mat *src, Mat *dst, int hgt, int way)
{
	int i;
	if (way == 0) {
		for (i = 0; i < hgt; ++i)
		{
			MatDerivationNoneActi(&src[i], &dst[i]);
		}

	}
	else if (way == 1) {
		for (i = 0; i < hgt; ++i)
		{
			MatDerivationSigmoid(&src[i], &dst[i]);
		}
	}
	else if (way == 2) {
		for (i = 0; i < hgt; ++i)
		{
			MatDerivationTanh(&src[i], &dst[i]);
		}
	}
	else if (way == 3) {
		for (i = 0; i < hgt; ++i)
		{
			MatDerivationRelu(&src[i], &dst[i]);
		}
	}
	else if (way == 4) {
		for (i = 0; i < hgt; ++i)
		{
			MatDerivationLeakyRelu(0.2f, &src[i], &dst[i]); //leak = 0.2
		}
	}
	else if (way == 5) {
		for (i = 0; i < hgt; ++i)
		{
			MatDerivationSoftmax(&src[i], &dst[i]);
		}
	}
	else {
		printf("error for MatActivate, please check ActiFsHidden  variable!\n");
	}
	return dst;
}

/************************************************************************/
/*                             ������������                               */
/************************************************************************/

/************************************************************************/
/*                              CNN����                                  */
/************************************************************************/

//--------------------------------ǰ�򴫲�--------------------------------//

// may need change
float LossFunction(Mat *src, Mat *dst, int Nstr_LossF) {
	if (Nstr_LossF == 0) {
		return MSE(src, dst);
	}
	else if (Nstr_LossF == 1) {
		return CrossEntropy(src, dst);
	}
	else {
		printf("error for Nstr_LossF, please check loss function variable!\n");
		return -1;
	}
}

// am struct and verified data struct
void inputCNN(float* data, CNN* cnn)
{
	// verified before
	int i, j, cnt = 0;
	for (i = 0; i < cnn->Layers[0].ActiMat->row; ++i)
	{
		for (j = 0; j < cnn->Layers[0].ActiMat->col; ++j)
		{
			(cnn->Layers[0].ActiMat->element[i])[j] = data[cnt];
			++cnt;
		}
	}
}

void NNFCforward(Layer* FCLayer, Mat* AM, int am_h)
{
	FCLayer->OriginMat = FCload(FCLayer->OriginMat, AM, am_h);
	FCLayer->SumMat = MatplusMulMat(FCLayer->SumMat, FCLayer->OutH, FCLayer->OriginMat, FCLayer->OutH, FCLayer->WeightMat);
	FCLayer->ActiMat = MatplusActivate(FCLayer->SumMat, FCLayer->ActiMat, 1, FCLayer->ActiFuncNum);
	FCLayer->DeriMat = MatplusDerivate(FCLayer->SumMat, FCLayer->DeriMat, FCLayer->OutH, FCLayer->ActiFuncNum);
}

void NNCVforward(Layer* CVLayer, Mat* AM, int am_h)
{
	CVLayer->OriginMat = MatPadReConstructBias(CVLayer->OriginMat, CVLayer->OutH, AM, am_h, CVLayer->KernelSize, CVLayer->Stride, CVLayer->PaddingMode);
	CVLayer->SumMat = MatplusMulMat(CVLayer->SumMat, CVLayer->OutH, CVLayer->OriginMat, CVLayer->OutH, CVLayer->WeightMat);
	CVLayer->ActiMat = MatplusActivate(CVLayer->SumMat, CVLayer->ActiMat, CVLayer->OutH, CVLayer->ActiFuncNum);
	CVLayer->DeriMat = MatplusDerivate(CVLayer->SumMat, CVLayer->DeriMat, CVLayer->OutH, CVLayer->ActiFuncNum);
}

void NNDCforward(Layer* DCLayer, Mat* AM, int am_h)
{
	DCLayer->OriginMat = MatplusMulMat(DCLayer->OriginMat, am_h, AM, am_h, DCLayer->KernelMat);
	DCLayer->SumMat = MatPadReConstruct(DCLayer->SumMat, DCLayer->OutH, DCLayer->OriginMat, am_h, DCLayer->KernelSize, DCLayer->Stride, 0);
	// kernelsize maybe wrong for unmatch HW
	DCLayer->ActiMat = MatplusActivate(DCLayer->ActiMat, DCLayer->ActiMat, DCLayer->OutH, DCLayer->ActiFuncNum);
	// softmax maybe logical wrong with this struct
	DCLayer->DeriMat = MatplusDerivate(DCLayer->SumMat, DCLayer->DeriMat, DCLayer->OutH, DCLayer->ActiFuncNum);
}

void NNMPforward(Layer* MPLayer, Mat* AM, int am_h)
{
	MPLayer->OriginMat = MatPadReConstruct(MPLayer->OriginMat, MPLayer->OutH, AM, am_h, MPLayer->KernelSize, MPLayer->Stride, MPLayer->PaddingMode);
	MPLayer->ActiMat = MaxChoose(MPLayer->ActiMat, MPLayer->OutH, &MPLayer->KernelMat, MPLayer->OriginMat, MPLayer->OutH);
}

void NNAPforward(Layer* APLayer, Mat* AM, int am_h)
{
	APLayer->OriginMat = MatPadReConstructBias(APLayer->OriginMat, APLayer->OutH, AM, am_h, APLayer->KernelSize, APLayer->Stride, APLayer->PaddingMode);
	APLayer->ActiMat = MatplusMulMat(APLayer->ActiMat, APLayer->OutH, APLayer->OriginMat, APLayer->OutH, APLayer->KernelMat);
}

void NNforwardLayer(Layer* layer, Mat* AM, int am_h)
{
	if (layer->LayerType == 0)
	{
		NNFCforward(layer, AM, am_h);
	}
	if (layer->LayerType == 1)
	{
		NNCVforward(layer, AM, am_h);
	}
	if (layer->LayerType == 2)
	{
		NNDCforward(layer, AM, am_h);
	}
	if (layer->LayerType == 3)
	{
		NNMPforward(layer, AM, am_h);
	}
	if (layer->LayerType == 4)
	{
		NNAPforward(layer, AM, am_h);
	}
}

float NNforwardSingle(CNN *cnn)
{
	int i;
	for (i = 1; i < cnn->HiddenLayerNum + 2; ++i)
	{
		NNforwardLayer(&cnn->Layers[i], cnn->Layers[i - 1].ActiMat, cnn->Layers[i - 1].OutH);
	}
	float loss = -1.f;
	loss = LossFunction(&cnn->Layers[cnn->HiddenLayerNum + 1].ActiMat[0], &cnn->OnehotMat, cnn->LossFuncNum);
	// should be transform to matplus and has maken sure that output layer is 1 of HW
	return loss;
}

//--------------------------------ǰ�򴫲�--------------------------------//

//--------------------------------���򴫲�--------------------------------//

Mat* NNFCbackward(Layer* FCLayer, Mat* fdel, int fdel_h)
{
	FCLayer->DeltaMat = MatplusProduct(FCLayer->DeltaMat, FCLayer->DeriMat, FCLayer->DeltaMat, FCLayer->OutH);
	fdel = MatplusMulMat_T_unmatch(fdel, fdel_h, FCLayer->DeltaMat, FCLayer->OutH, FCLayer->KernelMat);
	return fdel;
}

void NNFCCVNabla(Layer* FCLayer, int BatchSize)
{
	FCLayer->NablaMat = Matplus_TMulMatplusSumDes(&FCLayer->NablaMat, FCLayer->OriginMat, FCLayer->OutH, FCLayer->DeltaMat, FCLayer->OutH, BatchSize);
}

Mat* NNCVbackward(Layer* CVLayer, Mat* fdel, int fdel_h)
{
	CVLayer->DeltaMat = MatplusProduct(CVLayer->DeltaMat, CVLayer->DeriMat, CVLayer->DeltaMat, CVLayer->OutH);
	DumpMatplusHWCo(CVLayer->DeltaMat, CVLayer->OutH);// ok
	Mat* TmpMat = (Mat*)malloc(CVLayer->OutH * sizeof(Mat));
	int i;
	for (i = 0; i < CVLayer->OutH; ++i)
	{
		MatCreate(&TmpMat[i], CVLayer->OriginMat[0].row, CVLayer->OriginMat[0].col - 1);
	}
	TmpMat = MatplusMulMat_T(TmpMat, CVLayer->OutH, CVLayer->DeltaMat, CVLayer->OutH, CVLayer->KernelMat);
	DumpMatplusHWCo(TmpMat, CVLayer->OutH);// ok
	fdel = MatDeConstruct(fdel, fdel_h, TmpMat, CVLayer->OutH, CVLayer->KernelSize, CVLayer->Stride);
	DumpMatplusHWCo(fdel, fdel_h);// no
	for (i = 0; i < CVLayer->OutH; ++i)
	{
		MatDelete(&TmpMat[i]);
	}
	free(TmpMat);
	return fdel;
}

Mat* NNDCbackward(Layer* DCLayer, Mat* fdel, int fdel_h)
{
	DCLayer->DeltaMat = MatplusProduct(DCLayer->DeltaMat, DCLayer->DeriMat, DCLayer->DeltaMat, DCLayer->OutH);
	Mat* TmpMat = (Mat*)malloc(fdel_h * sizeof(Mat));
	int i;
	for (i = 0; i < DCLayer->OutH; ++i)
	{
		MatCreate(&TmpMat[i], DCLayer->OriginMat[0].row, DCLayer->OriginMat[0].col);
	}
	TmpMat = MatPadReConstruct(TmpMat, fdel_h, DCLayer->DeltaMat, DCLayer->OutH, DCLayer->KernelSize, DCLayer->Stride, 0);
	fdel = MatplusMulMat_T(fdel, fdel_h, TmpMat, fdel_h, DCLayer->KernelMat);
	for (i = 0; i < DCLayer->OutH; ++i)
	{
		MatDelete(&TmpMat[i]);
	}
	free(TmpMat);
	return fdel;
}// the order of KernelSize may cause error

void NNDCNabla(Layer* DCLayer ,int BatchSize)
{
	DCLayer->NablaMat = Matplus_TMulMatplusSumDes(&DCLayer->NablaMat, DCLayer->DeltaMat, DCLayer->OutH, DCLayer->OriginMat, DCLayer->OutH, BatchSize);
}

Mat* NNMPbackward(Layer* MPLayer, Mat* fdel, int fdel_h)
{
	Mat* TmpMat = (Mat*)malloc(MPLayer->OutH * sizeof(Mat));
	int i;
	for (i = 0; i < MPLayer->OutH; ++i)
	{
		MatCreate(&TmpMat[i], MPLayer->OriginMat[0].row, MPLayer->OriginMat[0].col);
	}
	TmpMat = DeMaxChoose(TmpMat, MPLayer->OutH, &MPLayer->KernelMat, MPLayer->DeltaMat, MPLayer->OutH);
	fdel = MatDeConstruct(fdel, fdel_h, TmpMat, MPLayer->OutH, MPLayer->KernelSize, MPLayer->Stride);
	for (i = 0; i < MPLayer->OutH; ++i)
	{
		MatDelete(&TmpMat[i]);
	}
	free(TmpMat);
	return fdel;
}//?? deconstruct

Mat* NNAPbackward(Layer* APLayer, Mat* fdel, int fdel_h)
{
	Mat* TmpMat = (Mat*)malloc(APLayer->OutH * sizeof(Mat));
	int i;
	for (i = 0; i < APLayer->OutH; ++i)
	{
		MatCreate(&TmpMat[i], APLayer->OriginMat[0].row, APLayer->OriginMat[0].col);
	}
	TmpMat = MatplusMulMat_T(TmpMat, APLayer->OutH, APLayer->DeltaMat, APLayer->OutH, APLayer->KernelMat);
	fdel = MatDeConstruct(fdel, fdel_h, TmpMat, APLayer->OutH, APLayer->KernelSize, APLayer->Stride);
	for (i = 0; i < APLayer->OutH; ++i)
	{
		MatDelete(&TmpMat[i]);
	}
	free(TmpMat);
	return fdel;
}//?? deconstruct

void NNbackwardLayer(Layer* layer, Mat* fdel, int fdel_h, int BatchSize)
{
	if (layer->LayerType == 0)
	{
		NNFCbackward(layer, fdel, fdel_h);
		NNFCCVNabla(layer, BatchSize);
		return;
	}
	if (layer->LayerType == 1)
	{
		NNCVbackward(layer, fdel, fdel_h);
		NNFCCVNabla(layer, BatchSize);
		return;
	}
	if (layer->LayerType == 2)
	{
		NNDCbackward(layer, fdel, fdel_h);
		NNDCNabla(layer, BatchSize);
		return;
	}
	if (layer->LayerType == 3)
	{
		NNMPbackward(layer, fdel, fdel_h);
		return;
	}
	if (layer->LayerType == 4)
	{
		NNAPbackward(layer, fdel, fdel_h);
		return;
	}
}

void NNbackwardSingle(CNN* cnn, int BatchSize)
{
	int i;
	for (i = cnn->HiddenLayerNum + 1; i > 0; --i)
	{
		NNbackwardLayer(&cnn->Layers[i], cnn->Layers[i - 1].DeltaMat, cnn->Layers[i - 1].OutH, BatchSize);
	}
}

//--------------------------------���򴫲�--------------------------------//


int main()
{
	Mat *am = (Mat *)malloc(5 * sizeof(Mat));
	float cons[75];
	for (int i = 0; i < 75;++i)
	{
		cons[i] = 2.0 * (i + 1);
	}
	for (int i = 0; i < 5; ++i)
	{
		MatCreate(&am[i], 5, 3);
		MatSetVal(&am[i], cons + 15 * i);
	}
	//DumpMatplusHWC(am, 5);

	Layer testL;
	testL.LayerType = 1;
	testL.KernelSize[0] = 2;
	testL.KernelSize[1] = 2;
	testL.PaddingMode = 1;
	testL.Stride[0] = 2;
	testL.Stride[1] = 2;
	testL.ActiFuncNum = 4;
	testL.OutC = 8;
	testL.OutH = 3;
	testL.OutW = 3;

	SpaceCreateConvLayer(&testL, 5, 5, 3);
	DumpConvLayer(testL);
	WeightInit_ChooseWay(&testL.WeightMat, 2);
	MatReduceRow(&testL.WeightMat, &testL.KernelMat);
	DumpMat(testL.WeightMat);
	DumpMat(testL.KernelMat);
	NNCVforward(&testL, am, 5);
	DumpMatplusHWCo(testL.OriginMat, testL.OutH);
	DumpMatplusHWCo(testL.SumMat, testL.OutH);
	DumpMatplusHWCo(testL.ActiMat, testL.OutH);
	DumpMatplusHWCo(testL.DeriMat, testL.OutH);

	Mat *del = (Mat *)malloc(5 * sizeof(Mat));
	for (int i = 0; i < 5;++i)
	{
		MatCreate(&del[i], 5, 3);
	}
	float *fons = (float *)malloc(72 * sizeof(float));
	for (int i = 0; i < 72;++i)
	{
		fons[i] = 0.2 + i * 1.0 / 90;
	}
	MatSetVal(&testL.DeltaMat[0], fons);
	MatSetVal(&testL.DeltaMat[1], fons+24);
	MatSetVal(&testL.DeltaMat[2], fons+48);
	DumpMatplusHWCo(testL.DeltaMat, testL.OutH);

	NNCVbackward(&testL, del, 5);// stock here
	DumpMatplusHWCo(testL.DeltaMat, 3);
	DumpMatplusHWCo(del, 5);

	return 0;
}

// FC:
// FCLoad verified
// MatplusMulMat verified: watchout fc HW
// Deri 345 no matching res(solved: 34 is 0/1/a, 5 maybe so small):seem to be occasional
// HeInit make it small in SumMat, may be inited for each row be better:1619 should be col?
// forward
// backward from fc to cv will be wrong for unmatched fdel & deltamat in function MatplusMulMat_T : add MatplusMulMat_T_unmatch
/*
	Layer testL;
	testL.LayerType = 0;
	testL.KernelSize[0] = 2;
	testL.KernelSize[1] = 2;
	testL.PaddingMode = 1;
	testL.Stride[0] = 2;
	testL.Stride[1] = 2;
	testL.ActiFuncNum = 3;
	testL.OutC = 8;
	testL.OutH = 1;
	testL.OutW = 1;


	SpaceCreateFullConnLayer(&testL, 5, 5, 3);
	WeightInit_ChooseWay(&testL.WeightMat, 2);
	MatReduceRow(&testL.WeightMat, &testL.KernelMat);
	DumpMat(testL.WeightMat);
	DumpMat(testL.KernelMat);
	NNFCforward(&testL, am, 5);
	DumpFullConnLayer(testL);
	DumpMatplusHWCo(testL.OriginMat, 1);
	DumpMatplusHWCo(testL.SumMat, 1);
	DumpMatplusHWCo(testL.ActiMat, 1);
	DumpMatplusHWCo(testL.DeriMat, 1);

	Mat *del = (Mat *)malloc(5 * sizeof(Mat));
	for (int i = 0; i < 5;++i)
	{
		MatCreate(&del[i], 5, 3);
	}
	float *fons = (float *)malloc(8 * sizeof(float));
	for (int i = 0; i < 8;++i)
	{
		fons[i] = 0.3 + (7.0 * i) / 80;
	}
	MatSetVal(&testL.DeltaMat[0], fons);
	DumpMatplusHWCo(testL.DeltaMat, 1);

	NNFCbackward(&testL, del, 5);
	DumpMatplusHWCo(testL.DeltaMat, 1);
	DumpMatplusHWCo(del, 5);
*/
// CV:
// waiting for fc bw
/*

*/
